'use strict'
const fs = require('fs')

const request = require('request')

var initDb = require('./utils/initDb');
initDb();

var articles = require('./models/articles');

var Crawler = require("crawler");

var c = new Crawler({
    // 这个回调每个爬取到的页面都会触发
    maxConnections: 10,

    callback: function (error, res, done) {
        if (error) {

            console.log(error);

        } else {
            var $ = res.$;

            var list = $('.dw_table .el').toArray();


            list.forEach(element => {

                var href = $(element).find('.t2 a').attr('href');

                var post = $(element).find('.t1 a').text().replace(/[ \n\r]/g, '');
                var company = $(element).find('.t2 a').text().replace(/[ \n\r]/g, '');
                var location = $(element).find('.t3').text().replace(/[ \n\r]/g, '');
                var salary = $(element).find('.t4').text().replace(/[ \n\r]/g, '');
                var releaseTime = $(element).find('.t5').text().replace(/[ \n\r]/g, '');
               

                var c1 = new Crawler({
                    maxConnection: 10,
                    rateLimit: 2000,

                    callback: function (err, res1, done1) {

                        if (err) {

                            console.log(err);

                        } else {
                            var $ = res1.$;
                            var Content = $(".ltype").toArray();

                            Content.forEach(e => {

                                var properties = $(e).text().replace(/[|].*$/, '').replace(/[ \n\r]/g, '');
                                var scale = $(e).text().replace(/^.*?[|]/, '').replace(/[|].*$/, '').replace(/[ \n\r]/g, '');

                                

                                articles.create({ Post: post, Company: company, Location: location, Salary: salary, ReleaseTime: releaseTime, Properties: properties,  Scale: scale })

                            })
                        }

                        done1();
                    }
                });

                c1.queue(href);

            });
        }
        done();
    }
});

// 寻找规律并循环爬取URL，使用默认的callback
for (var a = 1; a <= 81; a += 1) {

    var url = 'https://search.51job.com/list/010000%252C020000%252C030200%252C040000%252C180200,000000,7501,00,9,99,%25E6%2595%25B0%25E6%258D%25AE%25E5%2588%2586%25E6%259E%2590%25E5%25B8%2588,2,' + a + '.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=';

    c.queue(url);
}

